import requests
import json
import time
import logging
import schedule
import threading
import subprocess
import psutil
import socket
import sys
import os
from datetime import datetime
from typing import Dict, List, Optional

# HTTP短信网关配置
HTTP_SMS_AVAILABLE = True

# 添加项目根目录到Python路径
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))

# 导入Config类
from config import Config

# 配置日志
logging.basicConfig(
    level=logging.INFO,
    format='%(asctime)s - %(levelname)s - %(message)s',
    handlers=[
        logging.FileHandler('monitor.log', encoding='utf-8'),
        logging.StreamHandler()
    ]
)
logger = logging.getLogger(__name__)

class ServiceMonitor:
    def __init__(self, config_file=None):
        # 加载配置文件
        self.config_file = config_file or os.path.join(os.path.dirname(__file__), "monitor_config.yaml")
        self.config = self._load_config()
        
        # 测试问题配置
        self.test_questions = self.config.get("test_questions", [
            {
                "service": "dify",
                "question": "如何下载轧差表",
                "expected_keywords": ["如何", "下载", "轧差表"]
            },
            {
                "service": "invoice",
                "question": "奉贤石油公司六月的发票信息",
                "expected_keywords": ["奉贤石油", "发票", "查询"]
            }
        ])
        
        # 短信报警配置
        sms_alert_config = self.config.get("sms_alert", {})
        http_sms_config = sms_alert_config.get("http_sms", {})
        sms_rate_limits_config = sms_alert_config.get("rate_limits", {})
        
        self.sms_config = {
            "enabled": sms_alert_config.get("enabled", True) and HTTP_SMS_AVAILABLE,
            "phone_numbers": sms_alert_config.get("phone_numbers", ["13945310088"]),
            "gateway_url": http_sms_config.get("gateway_url", "http://163.20.40.13:10082/smsBridge/wl/SmsServlet"),
            "timeout": http_sms_config.get("timeout", 10),
            "message_template": sms_alert_config.get("message_template", "【系统监控告警】\n服务: {service_name}\n时间: {alert_time}\n问题: {error_message}\n请及时处理！"),
            "max_alerts_per_hour": sms_rate_limits_config.get("max_alerts_per_hour", 3),
            "alert_cooldown_minutes": sms_rate_limits_config.get("alert_cooldown_minutes", 20)
        }
        
        # 自动重启配置
        services_config = self.config.get("services", {})
        self.restart_config = {
            "enabled": True,  # 是否启用自动重启
            "max_restart_attempts": 3,  # 最大重启尝试次数
            "restart_cooldown_minutes": 10,  # 重启冷却时间（分钟）
            "service_port": 8088,  # 服务端口
            "service_host": "163.20.60.211",  # 服务主机
            "restart_script_path": "/home/jiuyv/ccagent/main.py",  # 替换为实际的main.py绝对路径
            "python_path": "python"  # 替换为实际的python解释器绝对路径
        }
        
        # 重启状态跟踪
        self.restart_history = {}
        self.restart_counts = {}
        self.last_restart_time = {}
        
        # 短信报警状态跟踪
        self.sms_alert_counts = {}
        self.last_sms_alert_time = {}
        self.sms_alert_history = {}
    
    def _load_config(self):
        """加载配置文件"""
        try:
            import yaml
            with open(self.config_file, 'r', encoding='utf-8') as file:
                return yaml.safe_load(file)
        except ImportError:
            logger.warning("未安装PyYAML库，将使用默认配置")
            return {}
        except FileNotFoundError:
            logger.warning(f"配置文件 {self.config_file} 不存在，将使用默认配置")
            return {}
        except Exception as e:
            logger.error(f"加载配置文件异常: {str(e)}")
            return {}
        
    def test_dify_service(self) -> bool:
        """测试Dify服务"""
        try:
            from dify_service import call_dify_workflow
            
            test_query = "如何下载轧差表"
            # 使用特殊的监控标识符，确保每次都是真实调用而非缓存结果
            user_id = f"monitor_{int(time.time())}_no_cache"
            
            logger.info(f"测试Dify服务: {test_query}")
            # 添加监控标识参数，确保绕过缓存
            import asyncio
            try:
                loop = asyncio.get_event_loop()
            except RuntimeError:
                loop = asyncio.new_event_loop()
                asyncio.set_event_loop(loop)
            result = loop.run_until_complete(call_dify_workflow(test_query, user_id, is_monitor=True))
            
            if result and "answer" in result:
                answer = result["answer"]
                logger.info(f"Dify服务测试成功: {answer[:100]}...")
                return True
            else:
                error_msg = f"返回结果异常: {result}"
                logger.error(f"Dify服务测试失败: {error_msg}")
                self.send_sms_alert("Dify服务", error_msg)
                return False
                
        except Exception as e:
            error_msg = f"调用异常: {str(e)}"
            logger.error(f"Dify服务测试异常: {error_msg}")
            return False
    
    def _send_sms_via_http(self, to_phone: str, message: str) -> bool:
        """
        通过HTTP短信网关发送短信
        
        Args:
            to_phone: 收件人手机号
            message: 短信内容
            
        Returns:
            bool: 发送是否成功
        """
        try:
            # 获取HTTP短信网关配置
            gateway_url = self.sms_config.get("gateway_url", "http://163.20.40.13:10082/smsBridge/wl/SmsServlet")
            
            # 构建请求数据
            data = {
                "phoneNo": to_phone,
                "sms": message,
                "code": "UTF-8"
            }
            
            # 发送HTTP POST请求
            response = requests.post(gateway_url, data=data, timeout=self.sms_config.get("timeout", 10))
            
            # 检查响应状态
            if response.status_code == 200:
                logger.info(f"短信发送成功: {to_phone}")
                return True
            else:
                logger.error(f"短信发送失败: {to_phone}, HTTP状态码: {response.status_code}, 响应: {response.text}")
                return False
                
        except Exception as e:
            logger.error(f"短信发送失败: {to_phone}, 异常: {str(e)}")
            return False
            
    def send_sms_alert(self, service_name: str, error_message: str) -> bool:
        """发送短信报警"""
        try:
            # 检查短信报警是否启用
            if not self.sms_config.get("enabled", False):
                logger.info("短信报警未启用")
                return False
                
            # 检查报警频率限制
            current_time = datetime.now()
            if not self._should_send_sms_alert(service_name, current_time):
                logger.info(f"短信报警频率限制，跳过发送: {service_name}")
                return False
            
            # 获取短信配置
            phone_numbers = self.sms_config.get("phone_numbers", [])
            gateway_url = self.sms_config.get("gateway_url", "")
            message_template = self.sms_config.get("message_template", "【系统监控告警】\n服务: {service_name}\n时间: {alert_time}\n问题: {error_message}\n请及时处理！")
            
            if not phone_numbers or not gateway_url:
                logger.error("短信报警配置不完整")
                return False
            
            # 构建短信内容
            sms_content = message_template.format(
                service_name=service_name,
                alert_time=current_time.strftime('%Y-%m-%d %H:%M:%S'),
                error_message=error_message
            )
            
            logger.info(f"准备发送短信报警: {sms_content}")
            
            # 向所有手机号发送短信
            success_count = 0
            for phone_number in phone_numbers:
                if self._send_sms_via_http(phone_number, sms_content):
                    success_count += 1
            
            # 如果至少有一个发送成功，则认为整体成功
            if success_count > 0:
                logger.info(f"短信报警发送成功: {service_name}, 成功发送 {success_count}/{len(phone_numbers)} 条")
                self._record_sms_alert(service_name, True)
                return True
            else:
                logger.error(f"短信报警发送失败: {service_name}")
                self._record_sms_alert(service_name, False)
                return False
                
        except Exception as e:
            logger.error(f"发送短信报警异常: {str(e)}")
            self._record_sms_alert(service_name, False)
            return False
            
    def _should_send_sms_alert(self, service_name: str, current_time: datetime) -> bool:
        """检查是否应该发送短信报警（频率限制）"""
        try:
            # 获取频率限制配置
            max_alerts_per_hour = self.sms_config.get("max_alerts_per_hour", 3)
            alert_cooldown_minutes = self.sms_config.get("alert_cooldown_minutes", 20)
            
            # 检查冷却时间
            if service_name in self.last_sms_alert_time:
                time_diff = (current_time - self.last_sms_alert_time[service_name]).total_seconds() / 60
                if time_diff < alert_cooldown_minutes:
                    logger.info(f"短信报警冷却中，还需等待 {alert_cooldown_minutes - time_diff:.1f} 分钟")
                    return False
            
            # 检查每小时最大发送次数
            alert_key = f"{service_name}_{current_time.strftime('%Y-%m-%d_%H')}_sms"
            if alert_key in self.sms_alert_counts:
                if self.sms_alert_counts[alert_key] >= max_alerts_per_hour:
                    logger.warning(f"短信报警本小时发送次数已达上限 ({max_alerts_per_hour})，跳过发送")
                    return False
            
            return True
            
        except Exception as e:
            logger.error(f"检查短信报警频率限制异常: {str(e)}")
            # 出现异常时允许发送，确保不会因为检查逻辑错误而漏掉重要报警
            return True
    
    def _record_sms_alert(self, service_name: str, success: bool = True):
        """记录短信报警信息"""
        current_time = datetime.now()
        service_key = f"{service_name}_{current_time.strftime('%Y-%m-%d_%H')}_sms"
        
        # 更新短信报警计数
        self.sms_alert_counts[service_key] = self.sms_alert_counts.get(service_key, 0) + 1
        
        # 更新最后短信报警时间
        self.last_sms_alert_time[service_name] = current_time
        
        # 记录短信报警历史
        if service_name not in self.sms_alert_history:
            self.sms_alert_history[service_name] = []
        self.sms_alert_history[service_name].append({
            "time": current_time,
            "message": f"服务异常短信报警" if success else f"服务短信报警发送失败"
        })
        
        # 清理超过24小时的短信报警记录
        self._cleanup_old_sms_alerts()
    
    def _cleanup_old_sms_alerts(self):
        """清理超过24小时的短信报警记录"""
        current_time = datetime.now()
        
        for service_name in list(self.sms_alert_history.keys()):
            self.sms_alert_history[service_name] = [
                alert for alert in self.sms_alert_history[service_name]
                if (current_time - alert["time"]).total_seconds() < 86400  # 24小时
            ]
            
            if not self.sms_alert_history[service_name]:
                del self.sms_alert_history[service_name]
    
    def test_invoice_service(self) -> bool:
        """测试发票服务"""
        try:
            from invoice_service import call_invoice_service
            
            test_query = "奉贤石油公司六月的发票信息"
            phone_number = "13945310088"
            
            logger.info(f"测试发票服务: {test_query}")
            # 添加监控标识参数，确保绕过缓存
            import asyncio
            try:
                loop = asyncio.get_event_loop()
            except RuntimeError:
                loop = asyncio.new_event_loop()
                asyncio.set_event_loop(loop)
            result = loop.run_until_complete(call_invoice_service(test_query, phone_number, is_monitor=True))
            
            if result and "answer" in result:
                answer = result["answer"]
                logger.info(f"发票服务测试成功: {answer[:100]}...")
                return True
            else:
                error_msg = f"返回结果异常: {result}"
                logger.error(f"发票服务测试失败: {error_msg}")
                self.send_sms_alert("发票服务", error_msg)
                return False
                
        except Exception as e:
            error_msg = f"调用异常: {str(e)}"
            logger.error(f"发票服务测试异常: {error_msg}")
            return False
    
    def test_intent_analysis(self) -> bool:
        """测试意图分析接口"""
        try:
            import requests
            import json
            
            # 构建测试请求数据
            test_data = {
                "user_query": "如何下载轧差表",
                "phone_number": "13945310088",
                "is_monitor": True  # 标识为监控请求，确保绕过缓存
            }
            
            # 获取服务配置
            host = self.restart_config["service_host"]
            port = self.restart_config["service_port"]
            url = f"http://{host}:{port}/intent_analysis"
            
            logger.info(f"测试意图分析接口: {test_data['user_query']}")
            
            # 发送POST请求
            headers = {
                "Content-Type": "application/json",
                "X-API-Key": "a1b2c3d4-e5f6-7890-abcd-1234567890ab"  # 添加API密钥
            }
            response = requests.post(url, json=test_data, headers=headers, timeout=30)
            
            if response.status_code == 200:
                result = response.json()
                # 检查返回结果的结构，处理嵌套的JSON格式
                if "answer" in result:
                    answer = result["answer"]
                    logger.info(f"意图分析接口测试成功: {answer[:100]}...")
                    return True
                elif "result" in result and isinstance(result["result"], dict) and "answer" in result["result"]:
                    # 处理嵌套的JSON结构: {"result": {"answer": "..."}}
                    answer = result["result"]["answer"]
                    logger.info(f"意图分析接口测试成功: {answer[:100]}...")
                    return True
                else:
                    error_msg = f"返回结果格式异常: {result}"
                    logger.error(f"意图分析接口测试失败: {error_msg}")
                    self.send_sms_alert("意图分析接口", error_msg)
                    return False
            else:
                error_msg = f"HTTP状态码异常: {response.status_code}, 响应内容: {response.text}"
                logger.error(f"意图分析接口测试失败: {error_msg}")
                self.send_sms_alert("意图分析接口", error_msg)
                return False
                
        except Exception as e:
            error_msg = f"调用异常: {str(e)}"
            logger.error(f"意图分析接口测试异常: {error_msg}")
            self.send_sms_alert("意图分析接口", error_msg)
            return False
    
    def run_monitoring(self):
        """执行监控任务"""
        logger.info("开始执行监控任务...")
        
        services_tested = 0
        services_passed = 0
        
        # 检查服务端口状态
        port_available = self.check_service_port()
        if not port_available:
            logger.error("服务端口检查失败，尝试重启服务")
            restart_success = self.restart_service()
            if not restart_success:
                logger.error("服务重启失败，服务不可用")
                services_tested += 1  # 主服务计为测试但失败
                # 发送主服务严重故障报警
                self.send_sms_alert("主服务", "服务端口不可用且重启失败，请立即处理！")
            else:
                logger.info("服务重启成功，主服务恢复正常")
                services_passed += 1
                services_tested += 1  # 主服务计为测试且成功
        else:
            logger.info("服务端口检查正常")
            services_passed += 1
            services_tested += 1  # 主服务计为测试且成功
        
        # 测试依赖服务
        if self.test_dify_service():
            services_passed += 1
        services_tested += 1
        
        if self.test_invoice_service():
            services_passed += 1
        services_tested += 1
        
        # 测试意图分析接口
        if self.test_intent_analysis():
            services_passed += 1
        services_tested += 1
        
        # 汇总监控结果
        logger.info(f"监控完成: {services_passed}/{services_tested} 个服务正常")
        
        if services_passed < services_tested:
            failed_count = services_tested - services_passed
            error_msg = f"有 {failed_count} 个服务异常"
            logger.error(error_msg)
            # 如果主服务不可用，发送特殊报警
            if not port_available:
                self.send_sms_alert("主服务", "主服务端口不可用，依赖服务可能受影响")
        else:
            logger.info("所有服务运行正常")
        
        # 系统资源监控
        logger.info("开始系统资源监控...")
        
        # CPU使用率监控
        cpu_ok = self.monitor_cpu_usage()
        
        # 内存使用情况监控
        memory_ok = self.monitor_memory_usage()
        
        # 磁盘空间监控
        disk_ok = self.monitor_disk_space()
        
        # 网络流量监控
        network_ok = self.monitor_network_traffic()
        
        # 汇总系统资源监控结果
        system_resources_ok = all([cpu_ok, memory_ok, disk_ok, network_ok])
        
        if system_resources_ok:
            logger.info("所有系统资源监控正常")
        else:
            logger.warning("系统资源监控发现异常")
    
    def check_service_port(self) -> bool:
        """检查服务端口是否可用"""
        try:
            host = self.restart_config["service_host"]
            port = self.restart_config["service_port"]
            
            # 创建socket连接测试端口
            sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
            sock.settimeout(5)  # 5秒超时
            result = sock.connect_ex((host, port))
            sock.close()
            
            if result == 0:
                logger.info(f"服务端口 {host}:{port} 连接正常")
                return True
            else:
                logger.warning(f"服务端口 {host}:{port} 连接失败")
                return False
                
        except Exception as e:
            logger.error(f"检查服务端口异常: {str(e)}")
            return False
    
    def find_service_process(self) -> Optional[psutil.Process]:
        """查找正在运行的服务进程（优化版：优先使用ss命令）"""
        try:
            logger.info("开始查找服务进程...")
            script_name = os.path.basename(self.restart_config["restart_script_path"])
            port = self.restart_config["service_port"]
            
            # 方法1: 使用ss命令查找占用指定端口的进程（优先推荐）
            try:
                logger.info(f"尝试通过端口 {port} 使用ss命令查找服务进程...")
                result = subprocess.run(["ss", "-tlnp"], 
                                     capture_output=True, 
                                     text=True, 
                                     timeout=10)
                if result.returncode == 0:
                    import re
                    for line in result.stdout.split('\n'):
                        if f":{port}" in line:
                            # 提取PID信息，格式通常为 "pid=1234,"
                            match = re.search(r'pid=(\d+)', line)
                            if match:
                                pid = match.group(1)
                                try:
                                    process = psutil.Process(int(pid))
                                    # 验证进程是否属于当前用户
                                    if process.username() == os.getlogin():
                                        logger.info(f"通过ss找到服务进程: PID={pid}, 进程名={process.name()}, 所有者={process.username()}")
                                        return process
                                    else:
                                        logger.warning(f"进程PID={pid}不属于当前用户{os.getlogin()}, 跳过处理")
                                except (psutil.NoSuchProcess, ValueError):
                                    continue
            except (subprocess.TimeoutExpired, FileNotFoundError) as e:
                logger.warning(f"使用ss命令查找进程失败: {str(e)}")
            
            # 方法2: 使用netstat命令（备选）
            try:
                logger.info(f"尝试通过端口 {port} 使用netstat命令查找服务进程...")
                result = subprocess.run(["netstat", "-tlnp"], 
                                     capture_output=True, 
                                     text=True, 
                                     timeout=10)
                if result.returncode == 0:
                    for line in result.stdout.split('\n'):
                        if f":{port}" in line:
                            parts = line.strip().split()
                            if len(parts) >= 7:
                                pid_info = parts[-1]
                                if '/' in pid_info:
                                    pid = pid_info.split('/')[0]
                                    try:
                                        process = psutil.Process(int(pid))
                                        # 验证进程是否属于当前用户
                                        if process.username() == os.getlogin():
                                            logger.info(f"通过netstat找到服务进程: PID={pid}, 进程名={process.name()}, 所有者={process.username()}")
                                            return process
                                        else:
                                            logger.warning(f"进程PID={pid}不属于当前用户{os.getlogin()}, 跳过处理")
                                    except (psutil.NoSuchProcess, ValueError):
                                        continue
            except (subprocess.TimeoutExpired, FileNotFoundError) as e:
                logger.warning(f"使用netstat命令查找进程失败: {str(e)}")
            
            # 方法3: 遍历所有进程（根据脚本名查找）
            logger.info("尝试通过进程遍历查找服务进程...")
            found_processes = []
            
            for proc in psutil.process_iter(['pid', 'name', 'cmdline', 'username']):
                try:
                    # 检查进程是否属于当前用户
                    if proc.info['username'] != os.getlogin():
                        continue
                        
                    # 检查进程命令行是否包含脚本名称
                    if proc.info['cmdline'] and any(script_name in cmd for cmd in proc.info['cmdline']):
                        found_processes.append(proc)
                        logger.info(f"找到匹配进程: PID={proc.info['pid']}, 命令行={' '.join(proc.info['cmdline'])}")
                except (psutil.NoSuchProcess, psutil.AccessDenied, psutil.ZombieProcess):
                    continue
            
            # 如果找到多个匹配进程，选择最新的一个
            if found_processes:
                if len(found_processes) > 1:
                    logger.info(f"找到 {len(found_processes)} 个匹配进程，将选择最新的一个")
                    # 按创建时间排序，选择最新的进程
                    found_processes.sort(key=lambda p: p.create_time(), reverse=True)
                    
                selected_process = found_processes[0]
                logger.info(f"通过进程遍历选择服务进程: PID={selected_process.pid}, 进程名={selected_process.name()}, 命令行={' '.join(selected_process.cmdline())}")
                return selected_process
            
            logger.warning("未找到正在运行的服务进程")
            return None
            
        except Exception as e:
            logger.error(f"查找服务进程异常: {str(e)}")
            return None
    
    def kill_service_process(self, process: psutil.Process) -> bool:
        """终止服务进程（最终优化版，适配当前环境）"""
        try:
            pid = process.pid
            process_name = process.name()
            logger.info(f"尝试终止服务进程: PID={pid}, 进程名={process_name}, 所有者={process.username()}")
            
            # 首先检查进程是否仍然存在
            if not psutil.pid_exists(pid):
                logger.info(f"服务进程 {pid} 已不存在")
                return True
            
            # 检查进程是否属于当前用户
            if process.username() != os.getlogin():
                logger.error(f"无权终止进程 {pid} (所有者: {process.username()}, 当前用户: {os.getlogin()})")
                return False
            
            # Windows系统处理逻辑
            if sys.platform == "win32":
                try:
                    logger.info(f"在Windows系统上尝试终止进程: PID={pid}, 进程名={process_name}")
                    result = subprocess.run(["taskkill", "/PID", str(pid), "/F", "/T"], 
                                         capture_output=True, 
                                         text=True, 
                                         timeout=10)
                    if result.returncode == 0:
                        logger.info(f"已使用taskkill强制终止服务进程: PID={pid}, 进程名={process_name}")
                        return True
                    else:
                        logger.warning(f"taskkill终止进程失败: PID={pid}, 错误={result.stderr}")
                except Exception as e:
                    logger.warning(f"taskkill命令执行异常: PID={pid}, 异常={str(e)}")
                
                # 使用psutil终止
                try:
                    logger.info(f"尝试使用psutil优雅终止进程: PID={pid}, 进程名={process_name}")
                    process.terminate()
                    process.wait(timeout=5)
                    logger.info(f"服务进程 {pid} ({process_name}) 已优雅终止")
                    return True
                except psutil.TimeoutExpired:
                    logger.warning(f"优雅终止超时，尝试强制终止进程: PID={pid}, 进程名={process_name}")
                    process.kill()
                    process.wait(timeout=3)
                    logger.info(f"服务进程 {pid} ({process_name}) 已强制终止")
                    return True
                except Exception as e:
                    logger.error(f"使用psutil终止进程异常: PID={pid}, 进程名={process_name}, 异常={str(e)}")
                    return False
            
            # Linux系统处理逻辑（完全适配当前环境）
            elif sys.platform.startswith("linux"):
                # 方法1: 尝试使用netstat+kill终止策略
                try:
                    port = self.restart_config["service_port"]
                    logger.info(f"在Linux系统上尝试使用netstat+kill策略终止进程: PID={pid}, 端口={port}")
                    result = subprocess.run(["netstat", "-tlnp"], 
                                         capture_output=True, 
                                         text=True, 
                                         timeout=10)
                    if result.returncode == 0:
                        for line in result.stdout.split('\n'):
                            if f":{port}" in line:
                                parts = line.strip().split()
                                if len(parts) >= 7:
                                    pid_info = parts[-1]
                                    if '/' in pid_info:
                                        netstat_pid = pid_info.split('/')[0]
                                        if netstat_pid == str(pid):
                                            # 找到匹配的进程，尝试使用kill命令终止
                                            logger.info(f"通过netstat找到匹配进程，尝试使用kill命令终止: PID={pid}")
                                            kill_result = subprocess.run(["kill", str(pid)], 
                                                                      capture_output=True, 
                                                                      text=True, 
                                                                      timeout=5)
                                            if kill_result.returncode == 0:
                                                logger.info(f"已使用netstat+kill策略终止服务进程: PID={pid}")
                                                # 等待进程终止
                                                for i in range(5):  # 最多等待5秒
                                                    time.sleep(1)
                                                    if not psutil.pid_exists(pid):
                                                        logger.info(f"进程 {pid} 已成功终止，耗时 {i+1} 秒")
                                                        return True
                                                logger.warning(f"进程 {pid} 未立即终止，尝试强制终止")
                                            else:
                                                logger.warning(f"netstat+kill策略执行返回非零状态: PID={pid}, 错误={kill_result.stderr}")
                except Exception as e:
                    logger.warning(f"netstat+kill策略执行异常: PID={pid}, 异常={str(e)}")
                
                # 方法2: 尝试优雅终止
                try:
                    logger.info(f"尝试使用kill命令优雅终止进程: PID={pid}")
                    result = subprocess.run(["kill", str(pid)], 
                                         capture_output=True, 
                                         text=True, 
                                         timeout=5)
                    if result.returncode == 0:
                        logger.info(f"已使用kill命令优雅终止服务进程: PID={pid}")
                        # 等待进程终止
                        for i in range(5):  # 最多等待5秒
                            time.sleep(1)
                            if not psutil.pid_exists(pid):
                                logger.info(f"进程 {pid} 已成功终止，耗时 {i+1} 秒")
                                return True
                        logger.warning(f"进程 {pid} 未立即终止，尝试强制终止")
                    else:
                        logger.warning(f"kill命令执行返回非零状态: PID={pid}, 错误={result.stderr}")
                except Exception as e:
                    logger.warning(f"kill命令执行异常: PID={pid}, 异常={str(e)}")
                
                # 方法3: 强制终止
                try:
                    logger.info(f"尝试使用kill -9强制终止进程: PID={pid}")
                    result = subprocess.run(["kill", "-9", str(pid)], 
                                         capture_output=True, 
                                         text=True, 
                                         timeout=5)
                    if result.returncode == 0:
                        logger.info(f"已使用kill -9强制终止服务进程: PID={pid}")
                        # 等待进程终止
                        for i in range(5):  # 最多等待5秒
                            time.sleep(1)
                            if not psutil.pid_exists(pid):
                                logger.info(f"强制终止进程 {pid} 成功，耗时 {i+1} 秒")
                                return True
                        logger.warning(f"强制终止后进程 {pid} 仍存在")
                    else:
                        logger.warning(f"kill -9命令执行返回非零状态: PID={pid}, 错误={result.stderr}")
                except Exception as e:
                    logger.warning(f"kill -9命令执行异常: PID={pid}, 异常={str(e)}")
                
                # 方法4: 使用psutil作为最后手段
                try:
                    if psutil.pid_exists(pid):
                        logger.info(f"尝试使用psutil终止进程: PID={pid}")
                        process.terminate()
                        try:
                            process.wait(timeout=3)
                            logger.info(f"psutil优雅终止进程 {pid} 成功")
                            return True
                        except psutil.TimeoutExpired:
                            logger.warning(f"psutil优雅终止超时，尝试强制终止进程: PID={pid}")
                            process.kill()
                            process.wait(timeout=2)
                            logger.info(f"psutil强制终止进程 {pid} 成功")
                            return True
                except psutil.AccessDenied:
                    logger.error(f"psutil访问被拒绝，无法终止进程 {pid}。解决方案：确认已配置sudoers文件允许jiuyvp1用户免密执行kill命令。sudoers配置方法：在/etc/sudoers.d/目录下创建配置文件，添加内容：jiuyvp1 ALL=(ALL) NOPASSWD: /bin/kill, /bin/kill -9")
                    return False
                except Exception as e:
                    logger.error(f"psutil终止进程异常: PID={pid}, 异常={str(e)}")
                    return False
            
            # 所有方法都失败
            logger.error(f"无法终止进程 {pid} ({process_name})")
            return False
                
        except Exception as e:
            logger.error(f"终止服务进程异常: {str(e)}")
            return False
    
    def restart_service(self) -> bool:
        """重启服务（最终优化版）"""
        if not self.restart_config["enabled"]:
            logger.info("自动重启功能未启用")
            return False
        
        current_time = datetime.now()
        service_key = f"main_service_{current_time.strftime('%Y-%m-%d_%H')}"
        
        # 检查重启频率限制
        if service_key in self.restart_counts:
            if self.restart_counts[service_key] >= self.restart_config["max_restart_attempts"]:
                logger.warning(f"服务本小时重启次数已达上限 ({self.restart_config['max_restart_attempts']})，跳过重启")
                return False
        
        # 检查重启冷却时间
        if "main_service" in self.last_restart_time:
            time_diff = (current_time - self.last_restart_time["main_service"]).total_seconds() / 60
            if time_diff < self.restart_config["restart_cooldown_minutes"]:
                logger.info(f"服务重启冷却中，还需等待 {self.restart_config['restart_cooldown_minutes'] - time_diff:.1f} 分钟")
                return False
        
        # 记录被终止的进程PID，用于后续日志记录
        terminated_pid = None
        
        try:
            logger.info("开始执行服务重启流程...")
            
            # 1. 查找并终止现有进程
            process = self.find_service_process()
            if process:
                terminated_pid = process.pid
                logger.info(f"找到需要终止的服务进程: PID={terminated_pid}")
                
                if not self.kill_service_process(process):
                    logger.error(f"终止现有服务进程失败: PID={terminated_pid}")
                    return False
                
                logger.info(f"成功终止服务进程: PID={terminated_pid}")
                
                # 等待端口释放
                logger.info("等待端口释放...")
                time.sleep(5)
            else:
                logger.info("未找到运行中的服务进程，将直接启动新服务")
            
            # 2. 检查端口是否已释放
            if self.check_service_port():
                logger.error("端口仍未释放，重启失败")
                return False
            
            logger.info("端口已成功释放")
            
            # 3. 启动新服务
            python_path = self.restart_config["python_path"]
            script_path = self.restart_config["restart_script_path"]
            
            # 验证脚本路径是否存在
            if not os.path.exists(script_path):
                error_msg = f"重启脚本不存在: {script_path}"
                logger.error(error_msg)
                self.send_sms_alert("服务重启", error_msg)
                return False
            
            logger.info(f"准备启动新服务: {python_path} {script_path}")
            
            # 启动新进程
            if sys.platform == "win32":
                startupinfo = subprocess.STARTUPINFO()
                startupinfo.dwFlags |= subprocess.STARTF_USESHOWWINDOW
                process = subprocess.Popen(
                    [python_path, script_path],
                    creationflags=subprocess.CREATE_NEW_PROCESS_GROUP | subprocess.CREATE_NO_WINDOW,
                    startupinfo=startupinfo
                )
            else:
                # Linux使用nohup确保进程在后台稳定运行
                with open("service_output.log", "a") as outfile:
                    process = subprocess.Popen(
                        ["nohup", python_path, script_path, "&"],
                        stdout=outfile,
                        stderr=subprocess.STDOUT,
                        start_new_session=True
                    )
            
            new_pid = process.pid
            logger.info(f"新服务进程已启动: PID={new_pid}")
            
            # 4. 等待服务启动并检查端口
            logger.info("等待服务启动...")
            for i in range(30):  # 最多等待30秒
                time.sleep(1)
                if self.check_service_port():
                    logger.info(f"服务重启成功，端口 {self.restart_config['service_port']} 可用，新进程PID={new_pid}")
                    self._record_restart("main_service", terminated_pid, new_pid)
                    return True
            
            error_msg = f"服务启动超时，端口不可用，新进程PID={new_pid}"
            logger.error(error_msg)
            self.send_sms_alert("服务重启", error_msg)
            return False
            
        except Exception as e:
            error_msg = f"重启服务异常: {str(e)}"
            logger.error(error_msg)
            self.send_sms_alert("服务重启", error_msg)
            return False
    
    def _record_restart(self, service_name: str, terminated_pid=None, new_pid=None):
        """记录重启信息"""
        current_time = datetime.now()
        service_key = f"{service_name}_{current_time.strftime('%Y-%m-%d_%H')}"
        
        # 更新重启计数
        if service_key not in self.restart_counts:
            self.restart_counts[service_key] = 0
        self.restart_counts[service_key] += 1
        
        # 更新最后重启时间
        self.last_restart_time[service_name] = current_time
        
        # 记录重启历史，包含PID信息
        if service_name not in self.restart_history:
            self.restart_history[service_name] = []
            
        # 构建详细的重启消息
        restart_message = "服务自动重启"
        if terminated_pid is not None and new_pid is not None:
            restart_message = f"服务自动重启: 终止PID={terminated_pid}, 新建PID={new_pid}"
        elif terminated_pid is not None:
            restart_message = f"服务自动重启: 终止PID={terminated_pid}"
        elif new_pid is not None:
            restart_message = f"服务自动重启: 新建PID={new_pid}"
            
        self.restart_history[service_name].append({
            "time": current_time,
            "message": restart_message,
            "terminated_pid": terminated_pid,
            "new_pid": new_pid
        })
        
        # 记录详细的重启日志
        logger.info(f"记录服务重启: 服务={service_name}, 时间={current_time}, {restart_message}")
        
        # 清理超过24小时的重启记录
        self._cleanup_old_restarts()
        
        # 发送重启通知，包含PID信息
        alert_message = "服务已自动重启"
        if terminated_pid is not None and new_pid is not None:
            alert_message = f"服务已自动重启: 终止PID={terminated_pid}, 新建PID={new_pid}"
        elif terminated_pid is not None:
            alert_message = f"服务已自动重启: 终止PID={terminated_pid}"
        elif new_pid is not None:
            alert_message = f"服务已自动重启: 新建PID={new_pid}"
            
        self.send_sms_alert(service_name, alert_message)
    
    def _cleanup_old_restarts(self):
        """清理超过24小时的重启记录"""
        current_time = datetime.now()
        for service_name in list(self.restart_history.keys()):
            self.restart_history[service_name] = [
                restart for restart in self.restart_history[service_name]
                if (current_time - restart["time"]).total_seconds() < 86400  # 24小时
            ]
            
            if not self.restart_history[service_name]:
                del self.restart_history[service_name]
    
    def monitor_cpu_usage(self) -> bool:
        """监控CPU使用率"""
        try:
            # 检查系统资源监控是否启用
            system_monitoring_config = self.config.get("system_monitoring", {})
            if not system_monitoring_config.get("enabled", True):
                logger.info("系统资源监控未启用，跳过CPU使用率监控")
                return True
                
            # 检查CPU使用率监控是否启用
            cpu_config = system_monitoring_config.get("cpu", {})
            if not cpu_config.get("enabled", True):
                logger.info("CPU使用率监控未启用")
                return True
                
            threshold = cpu_config.get("usage_threshold", 90.0)  # 默认阈值90%
            
            # 获取CPU使用率
            cpu_percent = psutil.cpu_percent(interval=1)
            logger.info(f"当前CPU使用率: {cpu_percent}%")
            
            # 检查是否超过阈值
            if cpu_percent > threshold:
                error_msg = f"CPU使用率过高: {cpu_percent}% (阈值: {threshold}%)"
                logger.warning(error_msg)
                self.send_sms_alert("CPU使用率", error_msg)
                return False
            
            return True
            
        except Exception as e:
            error_msg = f"CPU使用率监控异常: {str(e)}"
            logger.error(error_msg)
            return False
    
    def monitor_memory_usage(self) -> bool:
        """监控内存使用情况"""
        try:
            # 检查系统资源监控是否启用
            system_monitoring_config = self.config.get("system_monitoring", {})
            if not system_monitoring_config.get("enabled", True):
                logger.info("系统资源监控未启用，跳过内存使用情况监控")
                return True
                
            # 检查内存使用情况监控是否启用
            memory_config = system_monitoring_config.get("memory", {})
            if not memory_config.get("enabled", True):
                logger.info("内存使用情况监控未启用")
                return True
                
            usage_threshold = memory_config.get("usage_threshold", 90.0)  # 默认阈值90%
            
            # 获取内存使用情况
            memory = psutil.virtual_memory()
            memory_percent = memory.percent
            logger.info(f"当前内存使用率: {memory_percent}%")
            
            # 检查是否超过阈值
            if memory_percent > usage_threshold:
                error_msg = f"内存使用率过高: {memory_percent}% (阈值: {usage_threshold}%)"
                logger.warning(error_msg)
                self.send_sms_alert("内存使用率", error_msg)
                return False
            

            
            return True
            
        except Exception as e:
            error_msg = f"内存使用率监控异常: {str(e)}"
            logger.error(error_msg)
            return False
    
    def monitor_disk_space(self) -> bool:
        """监控磁盘空间"""
        try:
            # 检查系统资源监控是否启用
            system_monitoring_config = self.config.get("system_monitoring", {})
            if not system_monitoring_config.get("enabled", True):
                logger.info("系统资源监控未启用，跳过磁盘空间监控")
                return True
                
            # 检查磁盘空间监控是否启用
            disk_config = system_monitoring_config.get("disk", {})
            if not disk_config.get("enabled", True):
                logger.info("磁盘空间监控未启用")
                return True
                
            usage_threshold = disk_config.get("usage_threshold", 90.0)  # 默认阈值90%
            
            # 获取所有磁盘分区
            disk_partitions = psutil.disk_partitions()
            
            # 检查每个磁盘分区
            all_disks_ok = True
            for partition in disk_partitions:
                try:
                    # 跳过CD-ROM和特殊文件系统
                    if 'cdrom' in partition.opts or partition.fstype == '':
                        continue
                        
                    usage = psutil.disk_usage(partition.mountpoint)
                    usage_percent = (usage.used / usage.total) * 100
                    free_space_gb = usage.free / (1024 ** 3)
                    
                    logger.info(f"磁盘 {partition.mountpoint}: 使用率 {usage_percent:.1f}%, 剩余空间 {free_space_gb:.2f}GB")
                    
                    # 检查使用率是否超过阈值
                    if usage_percent > usage_threshold:
                        error_msg = f"磁盘 {partition.mountpoint} 使用率过高: {usage_percent:.1f}% (阈值: {usage_threshold}%)"
                        logger.warning(error_msg)
                        self.send_sms_alert("磁盘空间", error_msg)
                        all_disks_ok = False
                    

                        
                except PermissionError:
                    logger.warning(f"无权限访问磁盘分区: {partition.mountpoint}")
                    continue
                except Exception as e:
                    logger.error(f"检查磁盘分区 {partition.mountpoint} 异常: {str(e)}")
                    continue
            
            return all_disks_ok
            
        except Exception as e:
            error_msg = f"磁盘空间监控异常: {str(e)}"
            logger.error(error_msg)
            return False
    
    def monitor_network_traffic(self) -> bool:
        """监控网络流量"""
        try:
            # 检查系统资源监控是否启用
            system_monitoring_config = self.config.get("system_monitoring", {})
            if not system_monitoring_config.get("enabled", True):
                logger.info("系统资源监控未启用，跳过网络流量监控")
                return True
                
            # 检查网络流量监控是否启用
            network_config = system_monitoring_config.get("network", {})
            if not network_config.get("enabled", True):
                logger.info("网络流量监控未启用")
                return True
                
            max_upload_mbps = network_config.get("max_upload_mbps", 100.0)  # 默认最大上传速度100Mbps
            max_download_mbps = network_config.get("max_download_mbps", 100.0)  # 默认最大下载速度100Mbps
            
            # 获取网络IO统计信息
            net_io = psutil.net_io_counters()
            
            # 等待1秒后再次获取，计算速率
            time.sleep(1)
            net_io_after = psutil.net_io_counters()
            
            # 计算上传和下载速率（字节/秒）
            upload_speed = (net_io_after.bytes_sent - net_io.bytes_sent) / 1
            download_speed = (net_io_after.bytes_recv - net_io.bytes_recv) / 1
            
            # 转换为Mbps
            upload_speed_mbps = (upload_speed * 8) / (1024 ** 2)
            download_speed_mbps = (download_speed * 8) / (1024 ** 2)
            
            logger.info(f"当前网络速率 - 上传: {upload_speed_mbps:.2f}Mbps, 下载: {download_speed_mbps:.2f}Mbps")
            
            # 检查上传速率是否超过阈值
            if upload_speed_mbps > max_upload_mbps:
                error_msg = f"网络上传速率过高: {upload_speed_mbps:.2f}Mbps (阈值: {max_upload_mbps}Mbps)"
                logger.warning(error_msg)
                self.send_sms_alert("网络流量", error_msg)
                return False
            
            # 检查下载速率是否超过阈值
            if download_speed_mbps > max_download_mbps:
                error_msg = f"网络下载速率过高: {download_speed_mbps:.2f}Mbps (阈值: {max_download_mbps}Mbps)"
                logger.warning(error_msg)
                self.send_sms_alert("网络流量", error_msg)
                return False
            
            return True
            
        except Exception as e:
            error_msg = f"网络流量监控异常: {str(e)}"
            logger.error(error_msg)
            return False

def run_scheduler():
    """运行定时任务调度器"""
    monitor = ServiceMonitor()
    
    # 每5分钟执行一次监控
    schedule.every(5).minutes.do(monitor.run_monitoring)
    
    logger.info("监控服务启动，每5分钟检查一次服务状态")
    
    # 立即执行一次
    monitor.run_monitoring()
    
    # 运行调度器
    while True:
        schedule.run_pending()
        time.sleep(1)

if __name__ == "__main__":
    try:
        # 在后台线程中运行监控
        monitor_thread = threading.Thread(target=run_scheduler, daemon=True)
        monitor_thread.start()
        
        logger.info("监控服务已启动，按Ctrl+C停止")
        
        # 保持主线程运行
        while True:
            time.sleep(60)
            
    except KeyboardInterrupt:
        logger.info("监控服务停止")
    except Exception as e:
        logger.error(f"监控服务异常: {str(e)}")
        raise



